OpenCLIP Documentation

pip install open_clip_torch

import open_clip

model, _, preprocess = open_clip.create_model_and_transforms(
    'ViT-B-32',
    pretrained='laion2b_s34b_b79k'
)
model.eval()

import torch
from PIL import Image

# Get tokenizer
tokenizer = open_clip.get_tokenizer('ViT-B-32')

# Load and preprocess image
image = preprocess(Image.open("image.jpg")).unsqueeze(0)
text = tokenizer(["a cat", "a dog", "a bird"])

# Compute features
with torch.no_grad(), torch.autocast("cuda"):
    image_features = model.encode_image(image)
    text_features = model.encode_text(text)
    
    # Normalize features
    image_features /= image_features.norm(dim=-1, keepdim=True)
    text_features /= text_features.norm(dim=-1, keepdim=True)
    
    # Compute similarity and get predictions
    text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)

print("Predictions:", text_probs)

python -m open_clip_train.main \
    --train-data="/path/to/train_data.csv" \
    --val-data="/path/to/validation_data.csv" \
    --csv-img-key filepath \
    --csv-caption-key title \
    --warmup 10000 \
    --batch-size=128 \
    --lr=1e-3 \
    --wd=0.1 \
    --epochs=30 \
    --workers=8 \
    --model RN50